import requests
import js2xml
import socket
from lxml import etree
from DAO.DAO_exam_area import DAO_exam_area


class DATA_exam_area:
    # 爬取所有学校的专业链接
    def major_catalog_link(self):
        urls = ['http://college.wendu.com/index.php?m=university&c=search&a=search&page={}'.format(str(i)) for i in
                range(1, 71)]
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"}

        for url in urls:
            # print(url)
            socket.setdefaulttimeout(600)
            response = requests.get(url, headers=headers, timeout=600)
            response_html = etree.HTML(response.content.decode('utf-8'))
            schools = response_html.xpath("//a[@class='school-list-itemLeft']")
            for school in schools:
                school_name = school.xpath(".//div[2]//h3/text()")[0]
                major_link = "http://college.wendu.com/major_list-" + school.xpath("./@href")[0].split("-")[
                    1] + "-1-a-a"
                print(school_name, major_link)
                self.details_link(major_link)
                # break
            # break

    # 爬取某一个学校的专业的查看详情链接
    # 第二层
    def details_link(self, url):
        # url = 'http://college.wendu.com/major_list-1-1-a-a'
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        socket.setdefaulttimeout(600)
        response = requests.get(url, headers=headers, timeout=600)
        response_html = etree.HTML(response.content.decode())
        script_list = response_html.xpath("//script/text()")
        # print(response.content)
        # print(script_list[11])
        try:
            script_xml = js2xml.parse(script_list[11], encoding='utf-8', debug=False)
            # print(script_xml)
            script_tree = js2xml.pretty_print(script_xml)
            # print(script_tree)
            script_data_html = etree.HTML(script_tree)
            # print(etree.tostring(script_data_html).decode())
            div_all = script_data_html.xpath('//var[@name="datas"]/array/object')
            for div_one in div_all:
                unvid = div_one.xpath('.//property[@name="unvid"]/string/text()')[0]
                acadid = div_one.xpath('.//property[@name="acadid"]/string/text()')[0]
                majorid = div_one.xpath('.//property[@name="majorid"]/string/text()')[0]
                details_link = "http://college.wendu.com/major_detail-" + unvid + "-" + acadid + "-" + majorid
                maj_name = div_one.xpath('.//property[@name="major_name"]/string/text()')[0]
                maj_uni_name = response_html.xpath('//p[@class="where"]/a[2]/text()')[0]
                print(details_link, maj_name, maj_uni_name)
                self.exam_area(details_link)
                # break
        except Exception as e:
            print("爬取查看详情链接有错误》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》")
            print(e)

    # 在查看详情页面爬取 考试范围 具体数据，并插入数据库表 exam_area
    # 第三层
    def exam_area(self, url):
        # url = "http://college.wendu.com/major_detail-143-3504-137"
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        socket.setdefaulttimeout(600)
        response = requests.get(url, headers=headers, timeout=600)
        response_str = response.content.decode()
        response_html = etree.HTML(response_str)
        exa_year = response_html.xpath("//ul[@class='jb_content']/li[3]/ul/li[1]/text()")[0].split("：")[1]
        exa_maj_name = response_html.xpath("//p[@class='where']/text()")[2]
        exa_college_name = response_html.xpath("//ul[@class='clearfix']/li[2]/span/text()")[0]
        exa_uni_name = response_html.xpath("//ul[@class='clearfix']/li[1]/span/text()")[0]
        exam_area_one = response_html.xpath("//ul[@class='jb_content']/li[5]/div[2]/div")[0]
        e_a_div_list = exam_area_one.xpath("./text()")
        e_a_span_list = exam_area_one.xpath("./span/text()")
        e_a_p_list = exam_area_one.xpath("./p/text()")
        # print("div", e_a_div_list)
        # print("span", e_a_span_list)
        # print("p", e_a_p_list)
        exa_subjects = ""
        exa_subject_list = []
        e_a_div_one_one = ""
        dao = DAO_exam_area()
        # e_a_div_one ： div集合的一个元素
        # e_a_div_one_ ： div集合的一个元素,用空格符换行符分割后的集合元素（目的为去除换行符）
        # e_a_div_one_one ： div集合的一个元素（去除换行符）
        # 下面的for if if内容的作用：把div、span、p的集合元素全部集中到 exa_subject_list 中
        for e_a_div_one in e_a_div_list:
            # 如果div标签的 一个元素 里只有换行符，则不拼接它
            if len(e_a_div_one.split()) == 0:
                continue
            for e_a_div_one_ in e_a_div_one.split():
                e_a_div_one_one = e_a_div_one_one + e_a_div_one_
            exa_subject_list.append(e_a_div_one_one)
        if len(e_a_span_list) > 0:
            for e_a_span_one in e_a_span_list:
                exa_subject_list.append(e_a_span_one)
        if len(e_a_p_list) > 0:
            for e_a_p_one in e_a_p_list:
                exa_subject_list.append(e_a_p_one)
        # if内容的作用：把集合 exa_subject_list 字符串拼接到 exa_subjects 中
        if len(exa_subject_list) > 0:
            for exa_subject_one in exa_subject_list:
                exa_subjects = exa_subjects + exa_subject_one
        print("exa_subject_list", exa_subject_list)
        # print("exa_subjects", exa_subjects)
        # if内容的作用：分离出四科
        if len(exa_subjects.split("①")) == 2 and len(exa_subject_list) >= 1 and len(exa_subject_list) <= 5:
            exa_subjects = ""
            for i, e_s_l_one in enumerate(exa_subject_list):
                if i < 4:
                    exa_subjects = exa_subjects + e_s_l_one
            if len(exa_subjects.split("④")) == 2:
                exa_subject1 = exa_subjects.split("①")[1].split("②")[0]
                exa_subject2 = exa_subjects.split("②")[1].split("③")[0]
                exa_subject3 = exa_subjects.split("③")[1].split("④")[0]
                exa_subject4 = exa_subjects.split("④")[1]
            elif len(exa_subjects.split("④")) == 1:
                exa_subject1 = exa_subjects.split("①")[1].split("②")[0]
                exa_subject2 = exa_subjects.split("②")[1].split("③")[0]
                exa_subject3 = exa_subjects.split("③")[1]
                exa_subject4 = ""
            exa_subject1 = exa_subject1.replace('?', '')
            exa_subject2 = exa_subject2.replace('?', '')
            exa_subject3 = exa_subject3.replace('?', '')
            exa_subject4 = exa_subject4.replace('?', '')
            print("exa_subject1-----", exa_subject1, "-------")
            print("exa_subject2-----", exa_subject2, "-------")
            print("exa_subject3-----", exa_subject3, "-------")
            print("exa_subject4-----", exa_subject4, "-------")
            print(exa_year, ',', exa_maj_name, ',',  exa_college_name, ',',  exa_uni_name, ',',  exa_subject1, ',',  exa_subject2, ',',  exa_subject3, ',',
                  exa_subject4)
            dao.add(exa_year, exa_maj_name, exa_college_name, exa_uni_name, exa_subject1, exa_subject2, exa_subject3,
                    exa_subject4)
        # print("exa_subjects    ：", exa_subjects)
        # elif内容的作用：分离不出来的，放到科一
        elif exa_subjects is not "" and len(exa_subject_list) > 0:
            print("1111111111111111111111115555555555555555555555555")
            print("exa_subjects    ：", exa_subjects)
            print(exa_year, exa_maj_name, exa_college_name, exa_uni_name, exa_subjects)
            dao.add(exa_year, exa_maj_name, exa_college_name, exa_uni_name, exa_subjects, '', '', '')
        # ① ② ③ ④    "①", "②", "③", "④"


if __name__ == '__main__':
    print("1、爬取所有院校 研究方向 数据")
    print("2、爬取一个专业 研究方向 数据")
    print("3、清空数据库")
    print("4、输出所有数据")
    res = int(input())
    if res == 1:
        # 爬取所有院校 招生简章 数据
        exam_area = DATA_exam_area()
        exam_area.major_catalog_link()
    elif res == 2:
        exam_area = DATA_exam_area()
        exam_area.exam_area("http://college.wendu.com/major_detail-1-156-44")
    elif res == 3:
        dao = DAO_exam_area()
        dao.truncate()
    elif res == 4:
        dao = DAO_exam_area()
        dao.sel_all()
